package detailpage.download;

import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Element;

import parse.news.listpage.MiningDataRegion;
import learnnet.httpclient.SimpleHttpClient;
import md.MD;
import miningDataRegion.DataRegion;

import com.liyuncong.application.commontools.DoNothing;
import com.liyuncong.application.commontools.FileOperate;
import com.liyuncong.application.commontools.FileTools;

public class Downloader {
	private final String rootDir = "D:\\program\\bigdata"
			+ "\\people.com.cn";
	private Map<File, List<String>> dirUrlPair = new HashMap<>();
	private Map<String, Integer> failedTimes = new HashMap<>();
	private int maxAttemp = 3;
	private List<String> failedDownload = new LinkedList<>();
	{
		File root = new File(rootDir);
		FileTools.traverse(root, new FileOperate() {
			
			@Override
			public void action(File file) {
				List<String> value = new LinkedList<String>();
				File[] files = file.listFiles();
				for (File file2 : files) {
					if (file2.isFile() && file2.getName().equals("详情页url.txt")) {
						value.addAll(FileTools.
								readAllLinesFromFile(file2.toPath()));
					}
				}
				if (value.size() != 0) {
					dirUrlPair.put(file, value);
				}
			}
		}, new DoNothing());
	}
	
	public Downloader() {
		
	}
	
	public void download() {
		for(File key : dirUrlPair.keySet()) {
			System.out.println(key);
			
			List<String> value = new LinkedList<String>(dirUrlPair.get(key));
			for (int i = 0; i < value.size(); i++) {
				String url = value.get(i);
				try(OutputStream os = new FileOutputStream(key.toString() + "\\" + MD.MD5(url) + ".html")) {
					boolean success = SimpleHttpClient.requestGet(url,"gbk",  os, "utf-8");
					if (success) {
						System.out.println("download successfully: " + url);						
					} else {
						if (failedTimes.containsKey(url)) {
							int hasAttempTimes = failedTimes.get(url);
							if (hasAttempTimes <= maxAttemp) {
								value.add(url);
								failedTimes.put(url, hasAttempTimes + 1);
							} else {
								failedDownload.add(url);
							}
						} else {
							failedTimes.put(url, 1);
							value.add(url);
						}
						
						// 暂停一秒
						Thread.sleep(3000);
					}
				} catch (Exception e) {
				}
			}
		}
	}
	
	public void printFailedDownload() {
		System.out.println("failed to download: ");
		if (failedDownload.size() == 0) {
			System.out.println("All download successfully!");
		}
		for (String url : failedDownload) {
			System.out.println(url);
		}
	}
	
	public static void main(String[] args) {
		Downloader parser = new Downloader();
		parser.download();
		parser.printFailedDownload();
	}
}
